#It could be used to download the all micro papers.
#
use strict;
use LWP::Simple;
use HTML::TreeBuilder;
use WWW::Mechanize;
use Encode;

binmode(STDIN, ':encoding(utf8)');
binmode(STDOUT, ':encoding(utf8)');
binmode(STDERR, ':encoding(utf8)');

#micro的主页
my $micro_url = "http://portal.acm.org/toc.cfm?id=SERIES366&idx=SERIES366&type=series&coll=ACM&dl=ACM&part=series&WantType=Proceedings&title=MICRO&CFID=90880979&CFTOKEN=15336205";

my $mech = WWW::Mechanize->new();

$mech->get( $micro_url );
my @links = $mech->links(); 

#extract the micro links.
print "extract...\n";
my @urls = map { $_->url } @links; 
my @micro_links = grep { /^toc.cfm/i } @urls;
my @micro_links = map { "http://portal.acm.org/".$_  } @micro_links;

print join "\n",@micro_links;

print "downloading ...\n";

foreach my $url ( @micro_links ){
	download_one_year_micro_papers( $url );
}

#*********  子程序 *******

#下载某一年的micro文章
sub download_one_year_micro_papers{
	my $url = @_[0];
	
	print "downloading ",$url,"\n";

	my $content = get($url);
	my $tree = HTML::TreeBuilder->new();

	$tree->parse( $content );

	my $title_node = $tree->look_down('_tag','title');
	my $title = $title_node->as_text;
	
	open ABSTRACT,">>micro_abstract.txt";
	print ABSTRACT $title,"\n\n";
	close ABSTRACT;

	#将micro: micro'10 变成 micro_10
	$title =~ s/^MICRO: //g;
	$title =~ s/ /_/g;
	print "Title : ",$title,"\n";

	my $new_dir = $title;
	
	if( -e $new_dir ){
		print "The dir ",$new_dir,"is already exits.\n";
		return;
	}
	mkdir ($new_dir,755) || return; 
	chdir $new_dir;

	my @pdf_links = $tree->look_down('title', 'Pdf');
	foreach my $pdf ( @pdf_links ){
		print "pdf : ",$pdf->attr('href'),"\n";
	}	
	print "\n\n";

	#有的文章有序和结尾的pdf，它们以"http"开头
	my @matter_pdf = grep { /^http/i } @pdf_links;	
	print "pdf links : \n";
	print join "\n",@matter_pdf;
	foreach my $matter ( @matter_pdf ){
		download_paper( $matter );
	}

	#pdf link's parent is the micro paper's parent. 
	my @pdf_parent = map { $_->parent } @pdf_links;

	foreach my $parent ( @pdf_parent ){
		my $paper_node = $parent->look_down('class','medium-text');
		if( $paper_node ){
			my $url = "http://portal.acm.org/".($paper_node->attr('href'));	
			print "donwloading paper ",$url,"\n";
			download_micro_paper( $url );
			print "\n";
		}else{
			print "no paper node\n";
		}
	}
	print "\n\n";
	$tree->delete;

	#返回父目录
	chdir "..";
}

#下载某一篇micro paper
sub download_micro_paper{
	my $url = @_[0];

	open ABSTRACT,">>../micro_abstract.txt";

	my $content = get($url);
	my $tree = HTML::TreeBuilder->new();
	
	$tree->parse( $content );
	
	#print the title of the paper.
	my $title = $tree->look_down('_tag', 'strong');  
	my $title_text = "NULL";
	if( $title ) {
	      $title_text = $title->as_text;   
	} 
	#print the title of the paper.
	print ABSTRACT $title_text,"\n";
	
	#print the Abstract.
	#有两种形式的abstrat : 1.只有一段，则这个abstract在$abstract[2]->as_text中
	#                      2.如果$abstract[2]->as_text为空说明它有多个段落的abstract.
	my @abstract = $tree->look_down('class', 'abstract');  
	if( @abstract && $abstract[2] ) {
		my $abstract_text = $abstract[2]->as_text;
		if( $abstract_text =~ /\S/ ){
			print ABSTRACT "Abstract : \n";
			print ABSTRACT "\t",$abstract_text,"\n";
		}
		else{
			my $parent = ($abstract[2])->parent;
			my @child = $parent->look_down('_tag','p');
			shift @child;
			print ABSTRACT "Abstract : \n";
			foreach my $one_abstact ( @child ){
				print ABSTRACT "\t",$one_abstact->as_text,"\n";
			}
		}
	}else {
		print ABSTRACT "no abstract\n";
	}
	
	#download the paper
	my $download_paper = $tree->look_down('title','Pdf');
	if( $download_paper ){
		my $download_paper_url = "http://portal.acm.org/" . ($download_paper->attr('href'));
		print "URL : ",$download_paper_url,"\n";
		my $file = $title_text . ".pdf";
		print "filename : ",$file,"\n";
		download_paper( $download_paper_url,$file );
	}
	print ABSTRACT "URL : \n\t",$url,"\n";
	print ABSTRACT "\n";
	close ABSTRACT;
}

#download the paper according to the url and save it to a file.
#$url : the paper's link.
#$file : the name of the file.
sub download_paper {
	my ($url,$file) = @_;
	getstore($url,$file);
}
